{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found and verified text8.zip\n",
      "Data size 17005207\n",
      "Most common words (+UNK) [['UNK', 418391], ('the', 1061396), ('of', 593677), ('and', 416629), ('one', 411764)]\n",
      "Sample data [5244, 3083, 12, 6, 195, 2, 3134, 46, 59, 156] ['anarchism', 'originated', 'as', 'a', 'term', 'of', 'abuse', 'first', 'used', 'against']\n",
      "3083 originated -> 5244 anarchism\n",
      "3083 originated -> 12 as\n",
      "12 as -> 3083 originated\n",
      "12 as -> 6 a\n",
      "6 a -> 195 term\n",
      "6 a -> 12 as\n",
      "195 term -> 2 of\n",
      "195 term -> 6 a\n",
      "Initialized\n",
      "Average loss at step  0 :  285.263580322\n",
      "Nearest to into: interventionism, porting, capacities, anglo, invisibility, traverse, affective, tiling,\n",
      "Nearest to who: sherry, amethyst, qcd, ideogram, benefactor, dreadful, dear, diseased,\n",
      "Nearest to at: locates, lye, purified, deletion, bsp, dulles, infamy, nordic,\n",
      "Nearest to if: deposed, maker, sprites, academically, irritability, humphrey, shiraz, normalizing,\n",
      "Nearest to is: bierce, precluded, fugitive, overlooking, franklin, nullification, kant, fontane,\n",
      "Nearest to nine: sold, xliii, javelin, about, aleksandar, transcend, oi, jess,\n",
      "Nearest to world: cathari, adopters, beets, dissent, enciclopedia, investigates, hernando, polyphony,\n",
      "Nearest to this: hike, licks, patterning, ju, tuamotu, pepper, horthy, anthology,\n",
      "Nearest to be: footwear, bale, innocents, cotangent, adorno, roadster, fractals, transitioned,\n",
      "Nearest to may: incendiary, parents, seeded, waited, studd, lutetium, comply, bothnian,\n",
      "Nearest to while: undermines, litters, python, nowy, secretory, desegregation, ominous, overlooked,\n",
      "Nearest to in: purged, lovin, rentals, defected, metamorphic, unifil, ancyra, indicted,\n",
      "Nearest to other: metropolises, amniotic, rounding, resins, pathogenic, profiles, stimulant, former,\n",
      "Nearest to some: entire, alteration, corcovado, centered, vandalism, party, pleated, loosening,\n",
      "Nearest to often: aragorn, mayhem, lans, mansion, motorists, tame, brahmagupta, village,\n",
      "Nearest to first: seventeen, potters, germinate, logic, ry, tomino, vein, caesaris,\n",
      "Average loss at step  2000 :  114.24727221\n",
      "Average loss at step  4000 :  52.5593900058\n",
      "Average loss at step  6000 :  32.8075758768\n",
      "Average loss at step  8000 :  23.4674458418\n",
      "Average loss at step  10000 :  17.752705999\n",
      "Average loss at step  12000 :  14.0761369301\n",
      "Average loss at step  14000 :  11.9360701464\n",
      "Average loss at step  16000 :  9.85570686054\n",
      "Average loss at step  18000 :  8.38344122434\n",
      "Average loss at step  20000 :  8.04150149691\n",
      "Nearest to into: for, from, regent, and, absurd, of, israelite, operatorname,\n",
      "Nearest to who: and, he, also, covering, abacus, amethyst, photography, apatosaurus,\n",
      "Nearest to at: in, on, agouti, and, operatorname, warheads, fantasy, of,\n",
      "Nearest to if: satisfying, maker, coach, null, deposed, spur, attendant, national,\n",
      "Nearest to is: was, are, were, as, by, operatorname, dasyprocta, has,\n",
      "Nearest to nine: eight, six, zero, seven, five, agouti, four, circ,\n",
      "Nearest to world: operatorname, compared, directors, investigates, hello, run, garde, apatosaurus,\n",
      "Nearest to this: the, a, hike, it, and, which, phi, truetype,\n",
      "Nearest to be: have, was, is, by, spawning, christianity, sailor, invade,\n",
      "Nearest to may: archipelagoes, pornography, studd, comply, charges, machined, morgan, hbox,\n",
      "Nearest to while: and, for, muscle, methane, where, jersey, clarence, basins,\n",
      "Nearest to in: and, of, with, on, for, from, circ, nine,\n",
      "Nearest to other: argo, circ, starts, variance, basins, reginae, truffaut, zelazny,\n",
      "Nearest to some: the, seven, UNK, these, loosening, gollancz, transmission, savannah,\n",
      "Nearest to often: renunciation, mansion, endowed, mayhem, village, defense, bastiat, methane,\n",
      "Nearest to first: logic, mackenzie, samoa, signal, seventeen, top, ry, germinate,\n",
      "Average loss at step  22000 :  7.03446816981\n",
      "Average loss at step  24000 :  6.88687133253\n",
      "Average loss at step  26000 :  6.78251097941\n",
      "Average loss at step  28000 :  6.40689982224\n",
      "Average loss at step  30000 :  5.92629021835\n",
      "Average loss at step  32000 :  5.9661012882\n",
      "Average loss at step  34000 :  5.71966169751\n",
      "Average loss at step  36000 :  5.73221738565\n",
      "Average loss at step  38000 :  5.50686225879\n",
      "Average loss at step  40000 :  5.28580316842\n",
      "Nearest to into: from, for, zero, regent, on, israelite, trs, to,\n",
      "Nearest to who: he, and, also, they, zero, which, it, albury,\n",
      "Nearest to at: in, on, agouti, zero, from, operatorname, six, abet,\n",
      "Nearest to if: satisfying, spur, brahmin, maker, tuition, but, coach, since,\n",
      "Nearest to is: was, are, has, were, operatorname, zero, dasyprocta, agave,\n",
      "Nearest to nine: eight, seven, six, zero, five, four, agouti, three,\n",
      "Nearest to world: aargau, operatorname, directors, alia, investigates, compared, zero, development,\n",
      "Nearest to this: it, which, the, hike, that, zero, a, one,\n",
      "Nearest to be: have, was, been, by, is, were, are, zero,\n",
      "Nearest to may: can, would, could, will, pornography, bothnian, archipelagoes, must,\n",
      "Nearest to while: for, where, and, or, secretory, lymphoma, zero, muscle,\n",
      "Nearest to in: on, zero, from, circ, at, and, of, agouti,\n",
      "Nearest to other: profiles, many, callisto, stimulant, different, zero, circ, zelazny,\n",
      "Nearest to some: many, these, the, zero, gollancz, abet, savannah, officer,\n",
      "Nearest to often: renunciation, endowed, also, mansion, mayhem, there, bastiat, it,\n",
      "Nearest to first: logic, samoa, top, mackenzie, seventeen, potters, agouti, writs,\n",
      "Average loss at step  42000 :  5.36455445981\n",
      "Average loss at step  44000 :  5.25429992545\n",
      "Average loss at step  46000 :  5.22665030754\n",
      "Average loss at step  48000 :  5.2515968504\n",
      "Average loss at step  50000 :  4.99289732528\n",
      "Average loss at step  52000 :  5.01402707583\n",
      "Average loss at step  54000 :  5.1851027621\n",
      "Average loss at step  56000 :  5.02568614733\n",
      "Average loss at step  58000 :  5.05521826839\n",
      "Average loss at step  60000 :  4.95048535395\n",
      "Nearest to into: from, for, regent, to, israelite, absurd, trs, nine,\n",
      "Nearest to who: he, also, they, which, she, ursus, debtor, and,\n",
      "Nearest to at: in, on, agouti, five, marek, pulau, six, from,\n",
      "Nearest to if: when, but, spur, since, satisfying, brahmin, is, ursus,\n",
      "Nearest to is: was, are, has, operatorname, wct, kapoor, dasyprocta, viridian,\n",
      "Nearest to nine: eight, six, seven, four, five, zero, michelob, agouti,\n",
      "Nearest to world: operatorname, aargau, directors, kapoor, two, alia, development, nuances,\n",
      "Nearest to this: which, it, the, hike, that, truetype, marek, a,\n",
      "Nearest to be: have, been, was, by, were, is, not, are,\n",
      "Nearest to may: can, would, could, will, must, might, pornography, to,\n",
      "Nearest to while: for, where, and, lymphoma, or, however, gundam, ursus,\n",
      "Nearest to in: from, at, on, circ, and, viridian, of, agouti,\n",
      "Nearest to other: many, different, marek, profiles, callisto, circ, three, stimulant,\n",
      "Nearest to some: many, these, the, all, two, several, abet, officer,\n",
      "Nearest to often: also, renunciation, endowed, there, usually, still, generally, mayhem,\n",
      "Nearest to first: second, pulau, endgame, samoa, kapoor, logic, top, agouti,\n",
      "Average loss at step  62000 :  4.99247515774\n",
      "Average loss at step  64000 :  4.84028801703\n",
      "Average loss at step  66000 :  4.58116630733\n",
      "Average loss at step  68000 :  4.97990207922\n",
      "Average loss at step  70000 :  4.91599388289\n",
      "Average loss at step  72000 :  4.75363322234\n",
      "Average loss at step  74000 :  4.80360178339\n",
      "Average loss at step  76000 :  4.71966343582\n",
      "Average loss at step  78000 :  4.80549926364\n",
      "Average loss at step  80000 :  4.80793845129\n",
      "Nearest to into: from, busan, to, clo, callithrix, absurd, for, nine,\n",
      "Nearest to who: he, they, also, overline, which, she, never, and,\n",
      "Nearest to at: in, on, agouti, through, pulau, six, of, around,\n",
      "Nearest to if: when, but, spur, since, brahmin, busan, however, satisfying,\n",
      "Nearest to is: was, are, has, wct, operatorname, agave, busan, means,\n",
      "Nearest to nine: eight, seven, six, five, zero, four, michelob, microcebus,\n",
      "Nearest to world: operatorname, aargau, alia, directors, nuances, kapoor, mukherjee, development,\n",
      "Nearest to this: it, which, that, the, truetype, decoction, hike, there,\n",
      "Nearest to be: have, been, were, by, was, is, are, nine,\n",
      "Nearest to may: can, would, could, will, might, must, should, thaler,\n",
      "Nearest to while: where, for, however, and, lymphoma, busan, though, operatorname,\n",
      "Nearest to in: circ, from, agouti, of, at, michelob, microcebus, on,\n",
      "Nearest to other: many, different, marek, profiles, stimulant, callisto, zelazny, some,\n",
      "Nearest to some: many, these, all, several, the, abet, microcebus, most,\n",
      "Nearest to often: also, usually, generally, still, there, endowed, renunciation, not,\n",
      "Nearest to first: second, thaler, logic, designate, mackenzie, pulau, seventeen, top,\n",
      "Average loss at step  82000 :  4.76119516861\n",
      "Average loss at step  84000 :  4.7522432816\n",
      "Average loss at step  86000 :  4.78763222134\n",
      "Average loss at step  88000 :  4.74785924613\n",
      "Average loss at step  90000 :  4.72935908604\n",
      "Average loss at step  92000 :  4.6611855917\n",
      "Average loss at step  94000 :  4.7153989042\n",
      "Average loss at step  96000 :  4.68617888391\n",
      "Average loss at step  98000 :  4.58618298048\n",
      "Average loss at step  100000 :  4.70577920198\n",
      "Nearest to into: from, eight, in, with, through, clo, callithrix, absurd,\n",
      "Nearest to who: he, they, she, also, and, never, often, overline,\n",
      "Nearest to at: in, on, agouti, pulau, around, broch, marek, apatosaurus,\n",
      "Nearest to if: when, since, busan, brahmin, is, however, spur, but,\n",
      "Nearest to is: was, has, are, operatorname, wct, be, agave, dasyprocta,\n",
      "Nearest to nine: eight, seven, six, zero, five, four, michelob, three,\n",
      "Nearest to world: operatorname, topol, cathari, kapoor, mukherjee, aargau, alia, directors,\n",
      "Nearest to this: which, it, the, that, some, decoction, truetype, another,\n",
      "Nearest to be: been, have, is, was, were, by, are, not,\n",
      "Nearest to may: can, would, will, could, might, must, should, thaler,\n",
      "Nearest to while: for, however, and, where, lymphoma, but, busan, though,\n",
      "Nearest to in: at, during, on, circ, from, agouti, with, until,\n",
      "Nearest to other: different, many, marek, profiles, some, zech, callisto, circ,\n",
      "Nearest to some: many, these, several, the, all, microcebus, their, abet,\n",
      "Nearest to often: usually, also, generally, sometimes, still, there, commonly, endowed,\n",
      "Nearest to first: second, peacocks, top, mackenzie, last, designate, thaler, seventeen,\n"
     ]
    }
   ],
   "source": [
    "# encoding=utf8  \n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import collections\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "import zipfile\n",
    "\n",
    "import numpy as np\n",
    "from six.moves import urllib\n",
    "from six.moves import xrange  # pylint: disable=redefined-builtin\n",
    "import tensorflow as tf\n",
    "\n",
    "# Step 1: Download the data.\n",
    "url = 'http://mattmahoney.net/dc/'\n",
    "\n",
    "# 下载数据集\n",
    "def maybe_download(filename, expected_bytes):\n",
    "    \"\"\"Download a file if not present, and make sure it's the right size.\"\"\"\n",
    "    if not os.path.exists(filename):\n",
    "        filename, _ = urllib.request.urlretrieve(url + filename, filename)\n",
    "    # 获取文件相关属性\n",
    "    statinfo = os.stat(filename)\n",
    "    # 比对文件的大小是否正确\n",
    "    if statinfo.st_size == expected_bytes:\n",
    "        print('Found and verified', filename)\n",
    "    else:\n",
    "        print(statinfo.st_size)\n",
    "        raise Exception(\n",
    "            'Failed to verify ' + filename + '. Can you get to it with a browser?')\n",
    "    return filename\n",
    "\n",
    "filename = maybe_download('text8.zip', 31344016)\n",
    "\n",
    "# Read the data into a list of strings.\n",
    "def read_data(filename):\n",
    "    \"\"\"Extract the first file enclosed in a zip file as a list of words\"\"\"\n",
    "    with zipfile.ZipFile(filename) as f:\n",
    "        data = tf.compat.as_str(f.read(f.namelist()[0])).split()\n",
    "    return data\n",
    "\n",
    "# 单词表\n",
    "words = read_data(filename)\n",
    "\n",
    "# Data size\n",
    "print('Data size', len(words))\n",
    "\n",
    "# Step 2: Build the dictionary and replace rare words with UNK token.\n",
    "# 只留50000个单词，其他的词都归为UNK\n",
    "vocabulary_size = 50000\n",
    "\n",
    "def build_dataset(words, vocabulary_size):\n",
    "    count = [['UNK', -1]]\n",
    "    # extend追加一个列表\n",
    "    # Counter用来统计每个词出现的次数\n",
    "    # most_common返回一个TopN列表，只留50000个单词包括UNK  \n",
    "    # c = Counter('abracadabra')\n",
    "    # c.most_common()\n",
    "    # [('a', 5), ('r', 2), ('b', 2), ('c', 1), ('d', 1)]\n",
    "    # c.most_common(3)\n",
    "    # [('a', 5), ('r', 2), ('b', 2)]\n",
    "    # 前50000个出现次数最多的词\n",
    "    count.extend(collections.Counter(words).most_common(vocabulary_size - 1))\n",
    "    # 生成 dictionary，词对应编号, word:id(0-49999)\n",
    "    # 词频越高编号越小\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    # data把数据集的词都编号\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        if word in dictionary:\n",
    "            index = dictionary[word]\n",
    "        else:\n",
    "            index = 0  # dictionary['UNK']\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    # 记录UNK词的数量\n",
    "    count[0][1] = unk_count\n",
    "    # 编号对应词的字典\n",
    "    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reverse_dictionary\n",
    "\n",
    "# data 数据集，编号形式\n",
    "# count 前50000个出现次数最多的词\n",
    "# dictionary 词对应编号\n",
    "# reverse_dictionary 编号对应词\n",
    "data, count, dictionary, reverse_dictionary = build_dataset(words, vocabulary_size)\n",
    "del words  # Hint to reduce memory.\n",
    "print('Most common words (+UNK)', count[:5])\n",
    "print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])\n",
    "\n",
    "data_index = 0\n",
    "\n",
    "# Step 3: Function to generate a training batch for the skip-gram model.\n",
    "def generate_batch(batch_size, num_skips, skip_window):\n",
    "    global data_index\n",
    "    assert batch_size % num_skips == 0\n",
    "    assert num_skips <= 2 * skip_window\n",
    "    \n",
    "    batch = np.ndarray(shape=(batch_size), dtype=np.int32)\n",
    "    labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)\n",
    "    \n",
    "    span = 2 * skip_window + 1  # [ skip_window target skip_window ]\n",
    "    buffer = collections.deque(maxlen=span)\n",
    "    # [ skip_window target skip_window ]\n",
    "            # [ skip_window target skip_window ]\n",
    "                    # [ skip_window target skip_window ]\n",
    "            \n",
    "#     [0 1 2 3 4 5 6 7 8 9 ...]\n",
    "#            t     i  \n",
    "    # 循环3次\n",
    "    for _ in range(span):\n",
    "        buffer.append(data[data_index])\n",
    "        data_index = (data_index + 1) % len(data)\n",
    "    # 获取batch和labels\n",
    "    for i in range(batch_size // num_skips):\n",
    "        target = skip_window  # target label at the center of the buffer\n",
    "        targets_to_avoid = [skip_window]\n",
    "        # 循环2次，一个目标单词对应两个上下文单词\n",
    "        for j in range(num_skips):\n",
    "            while target in targets_to_avoid:\n",
    "                # 可能先拿到前面的单词也可能先拿到后面的单词\n",
    "                target = random.randint(0, span - 1)\n",
    "            targets_to_avoid.append(target)\n",
    "            batch[i * num_skips + j] = buffer[skip_window]\n",
    "            labels[i * num_skips + j, 0] = buffer[target]\n",
    "        buffer.append(data[data_index])\n",
    "        data_index = (data_index + 1) % len(data)\n",
    "    # Backtrack a little bit to avoid skipping words in the end of a batch\n",
    "    # 回溯3个词。因为执行完一个batch的操作之后，data_index会往右多偏移span个位置\n",
    "    data_index = (data_index + len(data) - span) % len(data)\n",
    "    return batch, labels\n",
    "\n",
    "# 打印sample data\n",
    "batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)\n",
    "for i in range(8):\n",
    "    print(batch[i], reverse_dictionary[batch[i]],\n",
    "        '->', labels[i, 0], reverse_dictionary[labels[i, 0]])\n",
    "\n",
    "# Step 4: Build and train a skip-gram model.\n",
    "batch_size = 128\n",
    "# 词向量维度\n",
    "embedding_size = 128  # Dimension of the embedding vector.\n",
    "skip_window = 1       # How many words to consider left and right.\n",
    "num_skips = 2         # How many times to reuse an input to generate a label.\n",
    "\n",
    "# We pick a random validation set to sample nearest neighbors. Here we limit the\n",
    "# validation samples to the words that have a low numeric ID, which by\n",
    "# construction are also the most frequent.\n",
    "valid_size = 16     # Random set of words to evaluate similarity on.\n",
    "valid_window = 100  # Only pick dev samples in the head of the distribution.\n",
    "# 从0-100抽取16个整数，无放回抽样\n",
    "valid_examples = np.random.choice(valid_window, valid_size, replace=False) \n",
    "# 负采样样本数\n",
    "num_sampled = 64    # Number of negative examples to sample.\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    # Input data.\n",
    "    train_inputs = tf.placeholder(tf.int32, shape=[batch_size])\n",
    "    train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])\n",
    "    valid_dataset = tf.constant(valid_examples, dtype=tf.int32)\n",
    "\n",
    "    # Ops and variables pinned to the CPU because of missing GPU implementation\n",
    "#     with tf.device('/cpu:0'):\n",
    "        # 词向量\n",
    "        # Look up embeddings for inputs.\n",
    "    embeddings = tf.Variable(\n",
    "        tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))\n",
    "    # embedding_lookup(params,ids)其实就是按照ids顺序返回params中的第ids行\n",
    "    # 比如说，ids=[1,7,4],就是返回params中第1,7,4行。返回结果为由params的1,7,4行组成的tensor\n",
    "    # 提取要训练的词\n",
    "    embed = tf.nn.embedding_lookup(embeddings, train_inputs)\n",
    "\n",
    "    # Construct the variables for the noise-contrastive estimation(NCE) loss\n",
    "    nce_weights = tf.Variable(\n",
    "        tf.truncated_normal([vocabulary_size, embedding_size],\n",
    "                        stddev=1.0 / math.sqrt(embedding_size)))\n",
    "    nce_biases = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "\n",
    "    # Compute the average NCE loss for the batch.\n",
    "    # tf.nce_loss automatically draws a new sample of the negative labels each\n",
    "    # time we evaluate the loss.\n",
    "    loss = tf.reduce_mean(\n",
    "        tf.nn.nce_loss(weights=nce_weights,\n",
    "                       biases=nce_biases,\n",
    "                       labels=train_labels,\n",
    "                       inputs=embed,\n",
    "                       num_sampled=num_sampled,   \n",
    "                       num_classes=vocabulary_size))\n",
    "\n",
    "    # Construct the SGD optimizer using a learning rate of 1.0.\n",
    "    optimizer = tf.train.GradientDescentOptimizer(1).minimize(loss)\n",
    "\n",
    "    # Compute the cosine similarity between minibatch examples and all embeddings.\n",
    "    norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))\n",
    "    normalized_embeddings = embeddings / norm\n",
    "    # 抽取一些常用词来测试余弦相似度\n",
    "    valid_embeddings = tf.nn.embedding_lookup(\n",
    "        normalized_embeddings, valid_dataset)\n",
    "    # valid_size == 16\n",
    "    # [16,1] * [1*50000] = [16,50000]\n",
    "    similarity = tf.matmul(\n",
    "        valid_embeddings, normalized_embeddings, transpose_b=True)\n",
    "\n",
    "    # Add variable initializer.\n",
    "    init = tf.global_variables_initializer()\n",
    "\n",
    "# Step 5: Begin training.\n",
    "num_steps = 100001\n",
    "final_embeddings = []\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    # We must initialize all variables before we use them.\n",
    "    init.run()\n",
    "    print(\"Initialized\")\n",
    "\n",
    "    average_loss = 0\n",
    "    for step in xrange(num_steps):\n",
    "        # 获取一个批次的target，以及对应的labels，都是编号形式的\n",
    "        batch_inputs, batch_labels = generate_batch(\n",
    "            batch_size, num_skips, skip_window)\n",
    "        feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}\n",
    "\n",
    "        # We perform one update step by evaluating the optimizer op (including it\n",
    "        # in the list of returned values for session.run()\n",
    "        _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)\n",
    "        average_loss += loss_val\n",
    "\n",
    "        # 计算训练2000次的平均loss\n",
    "        if step % 2000 == 0:\n",
    "            if step > 0:\n",
    "                average_loss /= 2000\n",
    "            # The average loss is an estimate of the loss over the last 2000 batches.\n",
    "            print(\"Average loss at step \", step, \": \", average_loss)\n",
    "            average_loss = 0\n",
    "    \n",
    "        # Note that this is expensive (~20% slowdown if computed every 500 steps)\n",
    "        if step % 20000 == 0:\n",
    "            sim = similarity.eval()\n",
    "            # 计算验证集的余弦相似度最高的词\n",
    "            for i in xrange(valid_size):\n",
    "                # 根据id拿到对应单词\n",
    "                valid_word = reverse_dictionary[valid_examples[i]]\n",
    "                top_k = 8  # number of nearest neighbors\n",
    "                # 从大到小排序，排除自己本身，取前top_k个值\n",
    "                nearest = (-sim[i, :]).argsort()[1:top_k + 1]\n",
    "                log_str = \"Nearest to %s:\" % valid_word\n",
    "                for k in xrange(top_k):\n",
    "                    close_word = reverse_dictionary[nearest[k]]\n",
    "                    log_str = \"%s %s,\" % (log_str, close_word)\n",
    "                print(log_str)\n",
    "    # 训练结束得到的词向量\n",
    "    final_embeddings = normalized_embeddings.eval()\n",
    "\n",
    "# Step 6: Visualize the embeddings.\n",
    "\n",
    "def plot_with_labels(low_dim_embs, labels, filename='tsne.png'):\n",
    "    assert low_dim_embs.shape[0] >= len(labels), \"More labels than embeddings\"\n",
    "    # 设置图片大小\n",
    "    plt.figure(figsize=(15, 15))  # in inches\n",
    "    for i, label in enumerate(labels):\n",
    "        x, y = low_dim_embs[i, :]\n",
    "        plt.scatter(x, y)\n",
    "        plt.annotate(label,\n",
    "                 xy=(x, y),\n",
    "                 xytext=(5, 2),\n",
    "                 textcoords='offset points',\n",
    "                 ha='right',\n",
    "                 va='bottom')\n",
    "\n",
    "    plt.savefig(filename)\n",
    "\n",
    "try:\n",
    "    from sklearn.manifold import TSNE\n",
    "    import matplotlib.pyplot as plt\n",
    "\n",
    "    tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact')# mac：method='exact'\n",
    "    # 画500个点\n",
    "    plot_only = 500\n",
    "    low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])\n",
    "    labels = [reverse_dictionary[i] for i in xrange(plot_only)]\n",
    "    plot_with_labels(low_dim_embs, labels)\n",
    "\n",
    "except ImportError:\n",
    "    print(\"Please install sklearn, matplotlib, and scipy to visualize embeddings.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nearest to one: two, seven, six, four, microcebus, five, three, eight,\n"
     ]
    }
   ],
   "source": [
    "# with tf.Session() as session:\n",
    "#     valid_word = \"one\"\n",
    "#     valid_examples = dictionary[valid_word]\n",
    "#     valid_dataset = tf.constant([valid_examples], dtype=tf.int32)\n",
    "#     valid_embeddings = tf.nn.embedding_lookup(final_embeddings, valid_dataset)\n",
    "#     similarity = tf.matmul(valid_embeddings, final_embeddings, transpose_b=True)\n",
    "#     sim = similarity.eval()\n",
    "\n",
    "#     top_k = 8  # number of nearest neighbors\n",
    "#     nearest = (-sim[0]).argsort()[1:top_k + 1]# 排除自己本身，从小到大排序\n",
    "#     log_str = \"Nearest to %s:\" % valid_word\n",
    "#     for k in xrange(top_k):\n",
    "#         close_word = reverse_dictionary[nearest[k]]\n",
    "#         log_str = \"%s %s,\" % (log_str, close_word)\n",
    "#     print(log_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
